{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-21T13:31:25.022339600Z",
     "start_time": "2023-11-21T13:31:25.016155200Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c552da9d-36f9-43d3-ae1f-c54f78d3ff2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from profiler.advisor.interface.interface import Interface\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from prettytable import PrettyTable, ALL\n",
    "from textwrap import fill"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57d17a21205c3c5e",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "# 集群调优分析\n",
    "## 1. 集群分析的数据准备\n",
    "首先我们当前支持PyTorch多卡大模型的集群分析，您需要输入集群分析的profiling_path路径，例如：  \n",
    "--{profiling_path}  \n",
    "    -- xxxx_ascend_pt  \n",
    "    -- xxxx_ascend_pt  \n",
    "    -- xxxx_ascend_pt  \n",
    "    ......  \n",
    "    -- xxxx_ascend_pt  \n",
    "里面每张卡的profiling文件都是ascend_pt结尾的文件。  \n",
    "\n",
    "## 2. 集群分析解决的问题  \n",
    "当前的功能主要有四项：  \n",
    "1）. 识别多卡间的计算慢卡（根据计算时间等推断）  \n",
    "2）. 识别多卡间的通信慢现象（根据通信链路的带宽判断）  \n",
    "3）. 对多卡间的计算算子进行统计展示（识别不同卡的算子差异）  \n",
    "4）. 展示集群流水并行图（根据时间轴展示多卡间的计算和通信时间）  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "36b7a24cc7ca5da2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-21T12:53:38.379699800Z",
     "start_time": "2023-11-21T12:53:38.363755900Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# EDIT THE PROFILING DATA PATH\n",
    "cluster_path = r\"YOUR PROFILING PATH\"\n",
    "interface = Interface(profiling_path=cluster_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf832ac2e0dfa30f",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## 1) 识别慢卡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "40aac93278dd6e34",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-21T12:53:41.815599700Z",
     "start_time": "2023-11-21T12:53:41.783393700Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]Cluster has been analyzed because of the existence of cluster analysis output directory.\n",
      "[INFO]Skip Cluster analyze backend.\n"
     ]
    }
   ],
   "source": [
    "slow_rank_result = interface.get_result(\"cluster\", \"slow_rank\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e943b2a-37a6-4db6-9e70-235d397f1d39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>rank_id</th>\n",
       "            <th>compute</th>\n",
       "            <th>communication</th>\n",
       "            <th>free</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>0</td>\n",
       "            <td>28976239.07999987</td>\n",
       "            <td>7586795.419999811</td>\n",
       "            <td>6836641.679994211</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>1</td>\n",
       "            <td>29012279.100000102</td>\n",
       "            <td>6984613.220000025</td>\n",
       "            <td>7388343.859991224</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>2</td>\n",
       "            <td>29019115.32300051</td>\n",
       "            <td>7489956.633000028</td>\n",
       "            <td>6881360.253991371</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>3</td>\n",
       "            <td>29027089.560000077</td>\n",
       "            <td>7963312.239999794</td>\n",
       "            <td>6389981.899993688</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>4</td>\n",
       "            <td>29044786.93699965</td>\n",
       "            <td>6533618.639000017</td>\n",
       "            <td>7780517.1539908135</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>5</td>\n",
       "            <td>29178186.259999853</td>\n",
       "            <td>7925184.420000028</td>\n",
       "            <td>6286867.999995028</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>6</td>\n",
       "            <td>29025331.189999904</td>\n",
       "            <td>6386639.90799992</td>\n",
       "            <td>7941798.704992032</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>7</td>\n",
       "            <td>29056803.304999545</td>\n",
       "            <td>7234444.826000024</td>\n",
       "            <td>7094608.035991492</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>8</td>\n",
       "            <td>31383314.980000228</td>\n",
       "            <td>3973806.6169999996</td>\n",
       "            <td>8017981.379989724</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>9</td>\n",
       "            <td>31360536.36200019</td>\n",
       "            <td>4757458.825000002</td>\n",
       "            <td>7277062.386991671</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>10</td>\n",
       "            <td>31381891.800000463</td>\n",
       "            <td>5276870.359999998</td>\n",
       "            <td>6731073.659992552</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>11</td>\n",
       "            <td>31387777.38000033</td>\n",
       "            <td>4727362.3000000045</td>\n",
       "            <td>7297578.339992355</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>12</td>\n",
       "            <td>31374132.74499977</td>\n",
       "            <td>5164443.388000004</td>\n",
       "            <td>6829798.933991944</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>13</td>\n",
       "            <td>31377800.178999804</td>\n",
       "            <td>4360616.283000001</td>\n",
       "            <td>7624691.509991412</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>14</td>\n",
       "            <td>31374658.360000316</td>\n",
       "            <td>4457099.620000001</td>\n",
       "            <td>7542724.319990785</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>15</td>\n",
       "            <td>31387255.527000006</td>\n",
       "            <td>5000860.905</td>\n",
       "            <td>6975264.115991174</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "+---------+--------------------+--------------------+--------------------+\n",
       "| rank_id |      compute       |   communication    |        free        |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    0    | 28976239.07999987  | 7586795.419999811  | 6836641.679994211  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    1    | 29012279.100000102 | 6984613.220000025  | 7388343.859991224  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    2    | 29019115.32300051  | 7489956.633000028  | 6881360.253991371  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    3    | 29027089.560000077 | 7963312.239999794  | 6389981.899993688  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    4    | 29044786.93699965  | 6533618.639000017  | 7780517.1539908135 |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    5    | 29178186.259999853 | 7925184.420000028  | 6286867.999995028  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    6    | 29025331.189999904 |  6386639.90799992  | 7941798.704992032  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    7    | 29056803.304999545 | 7234444.826000024  | 7094608.035991492  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    8    | 31383314.980000228 | 3973806.6169999996 | 8017981.379989724  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    9    | 31360536.36200019  | 4757458.825000002  | 7277062.386991671  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    10   | 31381891.800000463 | 5276870.359999998  | 6731073.659992552  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    11   | 31387777.38000033  | 4727362.3000000045 | 7297578.339992355  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    12   | 31374132.74499977  | 5164443.388000004  | 6829798.933991944  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    13   | 31377800.178999804 | 4360616.283000001  | 7624691.509991412  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    14   | 31374658.360000316 | 4457099.620000001  | 7542724.319990785  |\n",
       "+---------+--------------------+--------------------+--------------------+\n",
       "|    15   | 31387255.527000006 |    5000860.905     | 6975264.115991174  |\n",
       "+---------+--------------------+--------------------+--------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "slow_rank_data = slow_rank_result.get(\"slow_rank_analysis\")\n",
    "if slow_rank_data:\n",
    "    slow_rank_table = PrettyTable(slow_rank_data.get(\"headers\"))\n",
    "    for row in slow_rank_data.get(\"data\"):\n",
    "        row = [fill(str(element), width=80) for element in row]\n",
    "        slow_rank_table.add_row(row)\n",
    "    slow_rank_table.hrules = ALL\n",
    "    display(slow_rank_table[:16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "57a9b1c6-4127-47a2-8699-3c983950bd84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>problem</th>\n",
       "            <th>description</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>slow_rank_analysis</td>\n",
       "            <td>Computing      has some issues in the cluster,      because the max difference of Computing time<br>has reached 2411.538ms.  Communication      has some issues in the cluster,      because the max<br>difference of Communication time      has reached 3989.506ms.</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "+--------------------+--------------------------------------------------------------------------------------------------+\n",
       "|      problem       |                                           description                                            |\n",
       "+--------------------+--------------------------------------------------------------------------------------------------+\n",
       "| slow_rank_analysis | Computing      has some issues in the cluster,      because the max difference of Computing time |\n",
       "|                    | has reached 2411.538ms.  Communication      has some issues in the cluster,      because the max |\n",
       "|                    |                  difference of Communication time      has reached 3989.506ms.                   |\n",
       "+--------------------+--------------------------------------------------------------------------------------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "problems = slow_rank_result.get(\"problems\")\n",
    "headers = problems.get('headers')[:2]\n",
    "if problems: # 如果存在相关问题则获取相关问题检测描述及建议\n",
    "    problem_table = PrettyTable(headers)\n",
    "    for row in problems.get(\"data\"):\n",
    "        row = [fill(str(element), width=100) for element in row]\n",
    "        problem_table.add_row(row[:2])\n",
    "    display(problem_table)\n",
    "else:\n",
    "    print(\"There is no suggestion related to slow rank analysis.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3511befaff513e8e",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## 2）识别通信链路慢"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2a1e617d2a117125",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]Cluster has been analyzed because of the existence of cluster analysis output directory.\n",
      "[INFO]Skip Cluster analyze backend.\n"
     ]
    }
   ],
   "source": [
    "slow_link_result = interface.get_result(\"cluster\", \"slow_link\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c8bca314-a8da-4a5b-985a-c36f00154552",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>rank_id</th>\n",
       "            <th>RDMA bandwidth(GB/s)</th>\n",
       "            <th>RDMA size(mb)</th>\n",
       "            <th>RDMA time(ms)</th>\n",
       "            <th>SDMA bandwidth(GB/s)</th>\n",
       "            <th>SDMA size(mb)</th>\n",
       "            <th>SDMA time(ms)</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>9.7668</td>\n",
       "            <td>42507.3469439998</td>\n",
       "            <td>4352.225880000002</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>1</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>10.1653</td>\n",
       "            <td>42507.346775999795</td>\n",
       "            <td>4181.611080000001</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>2</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>10.471</td>\n",
       "            <td>42507.346775999795</td>\n",
       "            <td>4059.527798999999</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>3</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>9.9691</td>\n",
       "            <td>42507.346775999795</td>\n",
       "            <td>4263.9230400000015</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>4</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>9.1469</td>\n",
       "            <td>42507.346775999795</td>\n",
       "            <td>4647.202435000001</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>5</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>9.4663</td>\n",
       "            <td>42507.346775999795</td>\n",
       "            <td>4490.373999999999</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>6</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>9.5692</td>\n",
       "            <td>42507.346775999795</td>\n",
       "            <td>4442.106745000001</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>7</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>9.8444</td>\n",
       "            <td>42507.346775999795</td>\n",
       "            <td>4317.931616999999</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>8</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.895</td>\n",
       "            <td>42507.389952</td>\n",
       "            <td>2249.662369</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>9</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.9112</td>\n",
       "            <td>42507.39080800006</td>\n",
       "            <td>2247.7420159999997</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>10</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.7713</td>\n",
       "            <td>42507.39080800006</td>\n",
       "            <td>2264.48576</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>11</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.8389</td>\n",
       "            <td>42507.39080800006</td>\n",
       "            <td>2256.3606000000004</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>12</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.7687</td>\n",
       "            <td>42507.39080800006</td>\n",
       "            <td>2264.8021099999996</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>13</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.9717</td>\n",
       "            <td>42507.39080800006</td>\n",
       "            <td>2240.5713950000004</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>14</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.9226</td>\n",
       "            <td>42507.39080800006</td>\n",
       "            <td>2246.381839999999</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>15</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>0</td>\n",
       "            <td>18.8346</td>\n",
       "            <td>42507.39080800006</td>\n",
       "            <td>2256.8781</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "| rank_id | RDMA bandwidth(GB/s) | RDMA size(mb) | RDMA time(ms) | SDMA bandwidth(GB/s) |   SDMA size(mb)    |   SDMA time(ms)    |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    0    |          0           |       0       |       0       |        9.7668        |  42507.3469439998  | 4352.225880000002  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    1    |          0           |       0       |       0       |       10.1653        | 42507.346775999795 | 4181.611080000001  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    2    |          0           |       0       |       0       |        10.471        | 42507.346775999795 | 4059.527798999999  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    3    |          0           |       0       |       0       |        9.9691        | 42507.346775999795 | 4263.9230400000015 |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    4    |          0           |       0       |       0       |        9.1469        | 42507.346775999795 | 4647.202435000001  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    5    |          0           |       0       |       0       |        9.4663        | 42507.346775999795 | 4490.373999999999  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    6    |          0           |       0       |       0       |        9.5692        | 42507.346775999795 | 4442.106745000001  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    7    |          0           |       0       |       0       |        9.8444        | 42507.346775999795 | 4317.931616999999  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    8    |          0           |       0       |       0       |        18.895        |    42507.389952    |    2249.662369     |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    9    |          0           |       0       |       0       |       18.9112        | 42507.39080800006  | 2247.7420159999997 |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    10   |          0           |       0       |       0       |       18.7713        | 42507.39080800006  |     2264.48576     |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    11   |          0           |       0       |       0       |       18.8389        | 42507.39080800006  | 2256.3606000000004 |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    12   |          0           |       0       |       0       |       18.7687        | 42507.39080800006  | 2264.8021099999996 |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    13   |          0           |       0       |       0       |       18.9717        | 42507.39080800006  | 2240.5713950000004 |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    14   |          0           |       0       |       0       |       18.9226        | 42507.39080800006  | 2246.381839999999  |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+\n",
       "|    15   |          0           |       0       |       0       |       18.8346        | 42507.39080800006  |     2256.8781      |\n",
       "+---------+----------------------+---------------+---------------+----------------------+--------------------+--------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "slow_link_data = slow_link_result.get(\"slow_link_analysis\")\n",
    "if slow_link_data:\n",
    "    slow_link_table = PrettyTable(slow_link_data.get(\"headers\"))\n",
    "    for row in slow_link_data.get(\"data\"):\n",
    "        for i in range(len(row)):\n",
    "            row[i] = fill(str(row[i]), width=60)\n",
    "        slow_link_table.add_row(row)\n",
    "    slow_link_table.hrules = ALL\n",
    "    display(slow_link_table[:16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "77d6efa1-48e3-409f-82c4-3e2b3d868898",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>problem</th>\n",
       "            <th>description</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>slow_rank_analysis</td>\n",
       "            <td>Computing      has some issues in the cluster,      because the max difference of Computing time<br>has reached 2411.538ms.  Communication      has some issues in the cluster,      because the max<br>difference of Communication time      has reached 3989.506ms.</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>slow_link_analysis</td>\n",
       "            <td>SDMA bandwidth(GB/s):      The average is 14.332,      while the maximum  is 18.972GB/s      and the<br>minimum is 9.147GB/s.      the difference is 9.825GB/s.</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "+--------------------+------------------------------------------------------------------------------------------------------+\n",
       "|      problem       |                                             description                                              |\n",
       "+--------------------+------------------------------------------------------------------------------------------------------+\n",
       "| slow_rank_analysis |   Computing      has some issues in the cluster,      because the max difference of Computing time   |\n",
       "|                    |   has reached 2411.538ms.  Communication      has some issues in the cluster,      because the max   |\n",
       "|                    |                    difference of Communication time      has reached 3989.506ms.                     |\n",
       "| slow_link_analysis | SDMA bandwidth(GB/s):      The average is 14.332,      while the maximum  is 18.972GB/s      and the |\n",
       "|                    |                       minimum is 9.147GB/s.      the difference is 9.825GB/s.                        |\n",
       "+--------------------+------------------------------------------------------------------------------------------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "problems = slow_link_result.get(\"problems\")\n",
    "headers = problems.get('headers')[:2]\n",
    "if problems: # 如果存在相关问题则获取相关问题检测描述及建议\n",
    "    problem_table = PrettyTable(headers)\n",
    "    for row in problems.get(\"data\"):\n",
    "        row = [fill(str(element), width=100) for element in row]\n",
    "        problem_table.add_row(row[:2])\n",
    "    display(problem_table)\n",
    "else:\n",
    "    print(\"There is no suggestion related to slow link analysis.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce27a1d3-1354-45f7-88d8-dcb8e438b2b2",
   "metadata": {},
   "source": [
    "## 3) 分布式卡上的kernel算子统计展示"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "466a0f30-042c-492a-bbf2-a5a85b649f95",
   "metadata": {},
   "outputs": [],
   "source": [
    "from advisor_backend.interface import Interface\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "e05774e9-c47e-400f-8421-b4b71bcdcbc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "interface = Interface(cluster_path)\n",
    "dataset = interface.get_data('cluster', 'kernel')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "e95b6849-1738-4975-929f-734edff5d1c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rank id</th>\n",
       "      <th>Name</th>\n",
       "      <th>Input Shapes</th>\n",
       "      <th>Input Data Types</th>\n",
       "      <th>Output Shapes</th>\n",
       "      <th>Duration(us)_mean</th>\n",
       "      <th>Duration(us)_var</th>\n",
       "      <th>Duration(us)_max</th>\n",
       "      <th>Duration(us)_min</th>\n",
       "      <th>Duration(us)_count</th>\n",
       "      <th>Duration(us)_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Add100</td>\n",
       "      <td>\"4096,10880;4096,10880\"</td>\n",
       "      <td>FLOAT;FLOAT</td>\n",
       "      <td>\"4096,10880\"</td>\n",
       "      <td>478.210918</td>\n",
       "      <td>237.729252</td>\n",
       "      <td>721.420</td>\n",
       "      <td>449.80</td>\n",
       "      <td>1024</td>\n",
       "      <td>489687.980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Add102</td>\n",
       "      <td>\"21760;21760\"</td>\n",
       "      <td>FLOAT;FLOAT</td>\n",
       "      <td>\"21760\"</td>\n",
       "      <td>4.390391</td>\n",
       "      <td>0.011915</td>\n",
       "      <td>4.820</td>\n",
       "      <td>3.98</td>\n",
       "      <td>1024</td>\n",
       "      <td>4495.760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>Add106</td>\n",
       "      <td>\"21760,4096;21760,4096\"</td>\n",
       "      <td>FLOAT;FLOAT</td>\n",
       "      <td>\"21760,4096\"</td>\n",
       "      <td>933.504395</td>\n",
       "      <td>462.979321</td>\n",
       "      <td>1257.140</td>\n",
       "      <td>927.38</td>\n",
       "      <td>1024</td>\n",
       "      <td>955908.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Add111</td>\n",
       "      <td>\"4096,4096;4096,4096\"</td>\n",
       "      <td>FLOAT;FLOAT</td>\n",
       "      <td>\"4096,4096\"</td>\n",
       "      <td>91.267363</td>\n",
       "      <td>2.158275</td>\n",
       "      <td>97.120</td>\n",
       "      <td>85.12</td>\n",
       "      <td>1024</td>\n",
       "      <td>93457.780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>Add118</td>\n",
       "      <td>\"12288,4096;12288,4096\"</td>\n",
       "      <td>FLOAT;FLOAT</td>\n",
       "      <td>\"12288,4096\"</td>\n",
       "      <td>526.312012</td>\n",
       "      <td>1462.617511</td>\n",
       "      <td>787.780</td>\n",
       "      <td>424.24</td>\n",
       "      <td>1024</td>\n",
       "      <td>538943.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2513</th>\n",
       "      <td>15</td>\n",
       "      <td>trans_Cast_12</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>FLOAT</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>8.486495</td>\n",
       "      <td>0.060174</td>\n",
       "      <td>9.820</td>\n",
       "      <td>8.20</td>\n",
       "      <td>2048</td>\n",
       "      <td>17380.342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2514</th>\n",
       "      <td>15</td>\n",
       "      <td>trans_Cast_13</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>FLOAT</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>10.534564</td>\n",
       "      <td>0.166380</td>\n",
       "      <td>12.900</td>\n",
       "      <td>9.48</td>\n",
       "      <td>2048</td>\n",
       "      <td>21574.787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2515</th>\n",
       "      <td>15</td>\n",
       "      <td>trans_Cast_14</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>FLOAT</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>9.784551</td>\n",
       "      <td>0.295368</td>\n",
       "      <td>13.021</td>\n",
       "      <td>8.56</td>\n",
       "      <td>2048</td>\n",
       "      <td>20038.761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2516</th>\n",
       "      <td>15</td>\n",
       "      <td>trans_Cast_15</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>DT_BF16</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>8.342211</td>\n",
       "      <td>0.120471</td>\n",
       "      <td>10.220</td>\n",
       "      <td>7.86</td>\n",
       "      <td>2048</td>\n",
       "      <td>17084.848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2517</th>\n",
       "      <td>15</td>\n",
       "      <td>trans_Cast_16</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>DT_BF16</td>\n",
       "      <td>\"4096,1,1,128\"</td>\n",
       "      <td>9.507589</td>\n",
       "      <td>0.117111</td>\n",
       "      <td>11.681</td>\n",
       "      <td>9.18</td>\n",
       "      <td>2048</td>\n",
       "      <td>19471.543</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2518 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      rank id           Name             Input Shapes Input Data Types  \\\n",
       "0           0         Add100  \"4096,10880;4096,10880\"      FLOAT;FLOAT   \n",
       "1           0         Add102            \"21760;21760\"      FLOAT;FLOAT   \n",
       "2           0         Add106  \"21760,4096;21760,4096\"      FLOAT;FLOAT   \n",
       "3           0         Add111    \"4096,4096;4096,4096\"      FLOAT;FLOAT   \n",
       "4           0         Add118  \"12288,4096;12288,4096\"      FLOAT;FLOAT   \n",
       "...       ...            ...                      ...              ...   \n",
       "2513       15  trans_Cast_12           \"4096,1,1,128\"            FLOAT   \n",
       "2514       15  trans_Cast_13           \"4096,1,1,128\"            FLOAT   \n",
       "2515       15  trans_Cast_14           \"4096,1,1,128\"            FLOAT   \n",
       "2516       15  trans_Cast_15           \"4096,1,1,128\"          DT_BF16   \n",
       "2517       15  trans_Cast_16           \"4096,1,1,128\"          DT_BF16   \n",
       "\n",
       "       Output Shapes  Duration(us)_mean  Duration(us)_var  Duration(us)_max  \\\n",
       "0       \"4096,10880\"         478.210918        237.729252           721.420   \n",
       "1            \"21760\"           4.390391          0.011915             4.820   \n",
       "2       \"21760,4096\"         933.504395        462.979321          1257.140   \n",
       "3        \"4096,4096\"          91.267363          2.158275            97.120   \n",
       "4       \"12288,4096\"         526.312012       1462.617511           787.780   \n",
       "...              ...                ...               ...               ...   \n",
       "2513  \"4096,1,1,128\"           8.486495          0.060174             9.820   \n",
       "2514  \"4096,1,1,128\"          10.534564          0.166380            12.900   \n",
       "2515  \"4096,1,1,128\"           9.784551          0.295368            13.021   \n",
       "2516  \"4096,1,1,128\"           8.342211          0.120471            10.220   \n",
       "2517  \"4096,1,1,128\"           9.507589          0.117111            11.681   \n",
       "\n",
       "      Duration(us)_min  Duration(us)_count  Duration(us)_sum  \n",
       "0               449.80                1024        489687.980  \n",
       "1                 3.98                1024          4495.760  \n",
       "2               927.38                1024        955908.500  \n",
       "3                85.12                1024         93457.780  \n",
       "4               424.24                1024        538943.500  \n",
       "...                ...                 ...               ...  \n",
       "2513              8.20                2048         17380.342  \n",
       "2514              9.48                2048         21574.787  \n",
       "2515              8.56                2048         20038.761  \n",
       "2516              7.86                2048         17084.848  \n",
       "2517              9.18                2048         19471.543  \n",
       "\n",
       "[2518 rows x 11 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "27b75df4-792b-43dc-aa5c-d3c265642c1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存到csv查看， 可修改保存路径\n",
    "dataset.to_csv('cluster_kernel_details.csv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae45826394463cc4",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## 4) 展示集群流水并行图\n",
    "使用说明：  \n",
    "1）. 需要使用Ascend Torch Profiler采集数据，如果需要展示FP和BP需要将activities设置为采集CPU和NPU  \n",
    "2）. rank_ids为要展示的rank id列表，必选参数, 可视化顺序与rank_ids的顺序一致  \n",
    "3）. worker_num为多进程数量，可选参数，请根据机器配置调整，默认值为机器可用核心数的一半  \n",
    "4）. 如果没有采集CPU数据，则展示Stage和Bubble的流水图  \n",
    "5）. 生成的json文件可以在chrome trace中查看  \n",
    "\n",
    "示例图：\n",
    "![pipeline_view](../../profiler/test/resource/pipeline_view.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "baf66781eccfbca1",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Start to process 8 rank profiling data with 8 workers.\n",
      "[INFO] Pipline view data process finished, cost 98.48s.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "# rank_ids为要呈现的rank id列表，必选参数\n",
    "# 可以使用列表推导式生成需要的rank_ids，最终展示顺序和rank_ids的顺序一致\n",
    "# worker_num为多进程数量，可选参数，请根据机器配置调整，默认值为机器可用核心数的一半\n",
    "dataset = interface.get_data(\"cluster\", \"pipeline\", rank_ids=[0, 1, 2, 3, 4, 5, 6, 7], worker_num=8)\n",
    "\n",
    "# 保存json数据，在chrome trace中查看\n",
    "with open(\"./pipeline_view.json\", \"w\") as f:\n",
    "    json.dump(dataset.get(\"data\", []), f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f34ecf5-5c4a-4bc0-a761-e6338e534bac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
